#! /usr/bin/python
#coding=utf-8
import hashlib,re,sys
import gzip,urllib2,re,StringIO
from bs4 import BeautifulSoup
from sqlalchemy import *
from sqlalchemy import desc,asc
from sqlalchemy.orm import mapper, sessionmaker
default_encoding = 'utf-8'
if sys.getdefaultencoding() != default_encoding:
    reload(sys)
    sys.setdefaultencoding(default_encoding)
print sys.getdefaultencoding()

class Alllist(object):
    pass
class Centons(object):
    pass

class Yin:
    def __init__(self):
        self.engine =create_engine('mysql://root:123456@localhost/focal?charset=utf8',echo=True)
        #self.engine =create_engine('mysql://focal:focalweiming@localhost/focal?charset=utf8',echo=True)
        metadata = MetaData()
        alllist_table=Table('alllist2',metadata,
            Column('id',Integer, primary_key=True),
            Column('name',CHAR(255)),
            Column('url',CHAR(255)),
            Column('ordx',Integer),
        )
        centons_table=Table('centons2',metadata,
            Column('id',Integer, primary_key=True),
            Column('name',CHAR(255)),
            Column('url',CHAR(255)),
            Column('centon',Text),
            Column('ordx',Integer),
        )

        metadata.create_all(self.engine)
        mapper(Alllist,alllist_table)
        mapper(Centons,centons_table)
        Session = sessionmaker()
        Session.configure(bind=self.engine)
        self.db = Session()

    def getlist(self):
        listurl="http://www.qududu.com/book/69/69627/index.html"
        html=self.get_gzip(listurl)
        soup = BeautifulSoup(html)
        booklist=soup.find(name="div",attrs={"id":"chapter_list"})
        booklist=booklist.find_all(name="a")
        for l in booklist:
            odx=re.findall(r"\d+",l.get("href"))[0]
            if int(odx) < 13332351:
                continue
            dao=Alllist()
            dao.name=l.text
            dao.ordx=odx
            dao.url="http://www.qududu.com/book/69/69627/%s" % (l.get("href"))
            self.db.add(dao)
            self.db.flush()
            self.db.commit()

    def con(self):
        q=self.db.query(Alllist).filter(Alllist.ordx>13332351).filter(Alllist.ordx<14121717).group_by(Alllist.ordx).order_by(asc(Alllist.ordx)).all()
        for r in q:
            dao=Centons()
            html=self.get_gzip(r.url).decode("GBK", 'ignore')
            soup = BeautifulSoup(html)
            dao.centon=soup.find(name="div",attrs={"id":"text_area"})
            dao.ordx=r.ordx
            dao.name=r.name
            dao.url=r.url
            self.db.add(dao)
            self.db.flush()
            self.db.commit()

    def outtext(self):
        q=self.db.query(Centons).filter(Alllist.ordx>13332351).filter(Alllist.ordx<14121717).order_by(asc(Centons.ordx)).all()
        fobj=open('xiao688-870.txt','a')
        for r in q:
            r.centon=r.centon.strip('<div id="text_area">').strip('</div>').replace('<br/>','\n').replace('&nbsp;',' ').replace(u'  ','')
            r.centon=r.name+'--------------------\n'+r.centon+'\n'+r.name+'(END)\n\n'
            print r.centon
            fobj.write(r.centon)
        fobj.close()

    def get_gzip(self,url):
        header = {'Accept-Charset':'GBK,utf-8;q=0.7,*;q=0.3','User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16'}
        request = urllib2.Request(url,headers=header)
        request.add_header('Accept-encoding', 'gzip')
        opener = urllib2.build_opener()
        f = opener.open(request)
        print f
        isGzip = f.headers.get('Content-Encoding')
        if isGzip :
            compresseddata = f.read()
            compressedstream = StringIO.StringIO(compresseddata)
            gzipper = gzip.GzipFile(fileobj=compressedstream)
            data = gzipper.read()
        else:
            data = f.read()
        return data


if __name__ == '__main__':
    print "Prepare data. is come soon ..."
    cat=Yin()
    #cat.getlist()
    #cat.con()
    cat.outtext()